#!/usr/bin/env python
# -*- coding:utf-8 -*-

# 使用了线程库
import threading
# 队列
from Queue import Queue
# 解析库
from lxml import etree
# 请求处理
import requests
# json处理
import json
import time

class ThreadCrawl(threading.Thread):
    def __init__(self, threadName, pageQueue, dataQueue):
        # threading.Thread.__init__(self)
        super(ThreadCrawl, self).__init__()
        self.threadName = threadName
        self.pageQueue = pageQueue
        self.dataQueue = dataQueue
        self.headers = {"User-Agent" : "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}

    def run(self):
        print("启动" + self.threadName)
        while not CRAWL_EXIT:
            try:
                page = self.pageQueue.get(False)
                # https://www.qiushibaike.com/8hr/page/4/
                url = "https://www.qiushibaike.com/8hr/page/" + str(page) + "/"
                content = requests.get(url, headers=self.headers).text
                self.dataQueue.put(content)
            except:
                pass
        print("结束" + self.threadName)


class ParseCrawl(threading.Thread):
    def __init__(self, threadName, dataQueue, fileName, lock):
        super(ParseCrawl, self).__init__()
        self.threadName = threadName
        self.dataQueue = dataQueue
        self.fileName = fileName
        self.lock = lock

    def run(self):
        print("启动" + self.threadName)
        while not PARSE_EXIT:
            try:
                html = self.dataQueue.get(False)
                self.parse(html)
            except:
                pass
        print("结束" + self.threadName)

    def parse(self, html):
        html = etree.HTML(html)

        node_list = html.xpath('//div[contains(@id,"qiushi_tag_")]')

        # items ={}
        for node in node_list:
            # 用户名
            username = node.xpath('.//h2')[0].text
            # 内容
            content = node.xpath('./a/div[@class="content"]/span[1]')[0].text
            # 点赞
            vote = node.xpath('./div[@class="stats"]/span[@class="stats-vote"]/i')[0].text
            # 评论
            comment = node.xpath('./div[@class="stats"]/span[@class="stats-comments"]/a/i[@class="number"]')[0].text

            items = {
                "username": username,
                "content": content,
                "vote": vote,
                "comment": comment
            }

            with self.lock:
                self.fileName.write(json.dumps(items, ensure_ascii=False).encode('utf-8') + "\n")



CRAWL_EXIT = False
PARSE_EXIT = False


def main():
    # 10个页面
    pageQueue = Queue(10)
    for i in range(1, 101):
        pageQueue.put(i)

    # 采集结果的数据队列，参数为空表示不限制
    dataQueue = Queue()

    fileName = open("duanzi.json", "a")
    lock = threading.Lock()

    # 3个采集线程
    crawlList = ["采集线程1号",  "采集线程2号", "采集线程3号"]
    threadcrawl = []
    for threadName in crawlList:
        thread = ThreadCrawl(threadName, pageQueue, dataQueue)
        thread.start()
        threadcrawl.append(thread)

    # 3个解析线程
    parseList = ["解析线程1号", "解析线程2号", "解析线程3号"]
    threadparse = []
    for threadName in parseList:
        thread = ParseCrawl(threadName, dataQueue, fileName, lock)
        thread.start()
        threadparse.append(thread)

    while not pageQueue.empty():
        pass

    global CRAWL_EXIT
    CRAWL_EXIT = True

    print("pageQueue为空")

    for thread in threadcrawl:
        thread.join()
        print("1")

    while not dataQueue.empty():
        pass

    global PARSE_EXIT
    PARSE_EXIT = True

    for thread in threadparse:
        thread.join()
        print "2"

    with lock:
        # 关闭文件
        fileName.close()
    print "谢谢使用！"



if __name__ == "__main__":
    main()
